#include "pageLib.h"
#include "inicpp.hpp"
#include "../bin/simhash/Simhasher.hpp"
#include <fstream>
#include <filesystem>

using std::ifstream;
using std::ofstream;
using std::cerr;
using std::stringstream;
using std::regex;
using std::ios;
using std::cout;
using namespace simhash;
using namespace tinyxml2;

namespace fs = std::filesystem;


string removeEmptyLines(const string& input);


PageLib::PageLib(inicpp::IniManager* ini)
: _docId(0)
, _positionInFile(0)
, _ini(ini)
{
}

PageLib::~PageLib()
{

}

//移动构造
vector<RssItem> PageLib::parseRss(string filename){

    XMLDocument doc;
    if (doc.LoadFile(filename.c_str())!= XML_SUCCESS) {
        cerr << "Failed to load XML file" << "\n";
        return {};
    }

    XMLElement* root = doc.RootElement();
    if (root == nullptr) {
        cerr << "No root element found" << "\n";
        return {};
    }

    //临时存放单个xml文件的内容
    vector<RssItem> rss;

    // 遍历 <item> 元素
    for (XMLElement* itemElement = root->FirstChildElement("channel")->FirstChildElement("item"); \
        itemElement!= nullptr; \
        itemElement = itemElement->NextSiblingElement("item")) {
        // 在每个 <item> 中查找所需的元素
        XMLElement* linkElement = itemElement->FirstChildElement("link");
        XMLElement* titleElement = itemElement->FirstChildElement("title");
        XMLElement* descriptionElement = itemElement->FirstChildElement("description");
        
        // 获取数据保存到vector
        RssItem new_rss;
        new_rss.title = titleElement->GetText();
        new_rss.link = linkElement->GetText();

        //判断，以防止description为空
        if (descriptionElement) {
            string temp_descrip = descriptionElement->GetText();
            // 正则表达式去除 html 标签
            regex regex("<[^>]*>");
            new_rss.description = removeEmptyLines(regex_replace(temp_descrip, regex, ""));

            //去掉最后的换行符
            size_t pos = new_rss.description.find_last_not_of('\n');
            if (pos!= std::string::npos) {
                new_rss.description = new_rss.description.substr(0, pos + 1);
            }
        }     
        rss.push_back(new_rss);
        
    }
    return rss;
}

size_t calculateWriteSize(const string& title, const string& link, const string& description) {
    std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;
    std::u32string wideTitle = converter.from_bytes(title);
    std::u32string wideLink = converter.from_bytes(link);
    std::u32string wideDescription = converter.from_bytes(description);

    
    size_t strLength = sizeof("<doc>\n\t<docid></docid>\n\t<title></title>\n\t<link></link>\n \
                        \t<description></description>\n</doc>\n");
    size_t totalLength = wideTitle.length() + wideLink.length() + wideDescription.length() + strLength;
    return totalLength;
}



void PageLib::dump(ofstream& ofsRipepage, ofstream& offset, vector<RssItem> rss){

    //创建simhash对象
    Simhasher simhasher("../bin/dict/jieba.dict.utf8", "../bin/dict/hmm_model.utf8", "../bin/dict/idf.utf8", "../bin/dict/stop_words.utf8");
    for(auto & temp : rss){
        //doc的id
        _docId++;

        //网页去重
        stringstream webData;

        //计算每个doc的大小
        size_t docSize = 0;
        webData << "<doc>\n"
                << "\t<docid>" << _docId << "</docid>" << "\n"
                << "\t<title>" << temp.title << "</title>" << "\n"
                << "\t<link>" << temp.link << "</link>" << "\n"
                << "\t<description>" << temp.description << "</description>" << "\n"
                << "</doc>" << "\n";

        ofsRipepage << webData.str();

        //写入offset
        docSize = calculateWriteSize(temp.title, temp.link, temp.description);
        offset << _docId << " " << _positionInFile << " " << docSize << "\n";
        _positionInFile += docSize;
        
    }
}


void PageLib::analyze()
{
    //获取文件夹下所有文件
    _files = getFileNamesInDirectoryAndMove((*_ini)["rtsp"]["dirPath"]);

    //打开网页库文件
    ofstream ofsRipepage((*_ini)["rtsp"]["ripepage"], ios::app);
    if(!ofsRipepage){
        cerr << "file open fail" ;
    }
    //打开网页位置库文件
    ofstream ofsOffset((*_ini)["rtsp"]["offset"], ios::app);
    if(!ofsOffset){
        cerr << "file open fail" ;
    }

    
    //逐个文件处理并写入
    for(auto &file : _files)
    {
        string fullPath = (*_ini)["rtsp"]["dirPath"] + file;
        cout << fullPath << "\n";
        dump(ofsRipepage, ofsOffset, parseRss(fullPath));
    }

    ofsRipepage.close();
    ofsOffset.close();
}


//获取目录下所有文件名
vector<string> PageLib::getFileNamesInDirectory(const string& directoryPath) {
    vector<string> fileNames;
    try {
        for (const auto& entry : fs::directory_iterator(directoryPath)) {
            if (fs::is_regular_file(entry)) {
                fileNames.push_back(entry.path().filename().string());
            }
        }
    } catch (const fs::filesystem_error& e) {
        cerr << "Error accessing directory: " << e.what() << "\n";
    }
    return fileNames;
}

// 修改后的函数，返回值使用移动构造
vector<string> PageLib::getFileNamesInDirectoryAndMove(const string& directoryPath) {
    return vector<string>(getFileNamesInDirectory(directoryPath));
}

// 去除content和description的空行
string removeEmptyLines(const string &input) {
    stringstream ss(input);
    string line;
    string result;

    while (getline(ss, line)) {
        if (!line.empty()) {
            result += line + '\n';
        }
    }

    return result;
}
